from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

print "Program Begins..."
# key: term
# value: termID
temrANDTermIDDict = {}
inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithTermID"
inputFileHandler = open(inputFileName1,"r")
currentLine = inputFileHandler.readline()
while currentLine:
    currentLineElements = currentLine.strip().split(" ") 
    currentTermID = currentLineElements[0]
    currentTerm = currentLineElements[1]
    if currentTerm not in temrANDTermIDDict:
        temrANDTermIDDict[currentTerm] = currentTermID  
    currentLine = inputFileHandler.readline()
inputFileHandler.close()
print "len(temrANDTermIDDict): ",len(temrANDTermIDDict)

inputFileName2 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/tail5KResults_NEW_FORMAT_20140222Afternoon_middle_step1"
inputFileHandler = open(inputFileName2,"r")

outputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/tail5KResults_NEW_FORMAT_20140222Afternoon"
outputFileHandler = open(outputFileName,"w")

currentLine = inputFileHandler.readline()
while currentLine:
    currentLineElements = currentLine.strip().split(" ") 
    termID = temrANDTermIDDict[ currentLineElements[2] ]
    outputFileHandler.write(currentLineElements[0] + " " + currentLineElements[1] + " " + termID + " " + currentLineElements[3] + "\n")
    currentLine = inputFileHandler.readline()

print "Overall:"
print "inputFileName1:",inputFileName1
print "inputFileName2:",inputFileName2
print "outputFileName:",outputFileName
inputFileHandler.close()
outputFileHandler.close()
exit(1)



#dict1 = {}
#dict1["a"] = 2
#dict1["b"] = 2
#dict1["c"] = 5
#dict1["d"] = 3
 


#list1 = []
#list1.append("a")
#list1.append("c")
#list1.append("b")
#list1.sort(cmp=None, key=None, reverse=True)
#print list1
#list2 = []
#list2 = dict1.keys()
#list2.sort(cmp=None, key=None, reverse=True)
#print list2
# value = 2
# if value == 1:
#     print "1"
# elif value == 2:
#     print "2"

# a = 1
# b = 2
# print a + b
# n = 100
# a = n
# sum = 0
#for i in range(1,n+1):
#    sum = sum + i
# print sum
exit(1)

outputFileName = "/home/diaosi/web-search-engine-wei/polyIRIndexer/docID_num_of_postings_recorded_in_index_MappingTableForGov2Dataset"
outputFileHandler = open(outputFileName,"w")

inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_MappingTableForGov2Dataset_with_Xdoc_values_added_sorted_by_XdocValueUsingGoodTurningDividedByNumOfPostingsForEachDocument_sortedByTrecID"
inputFileHandler = open(inputFileName,"r")

currentLine = inputFileHandler.readline()
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    docID = currentLineElements[1]
    numOfPostingsInStringFormat = currentLineElements[2]
    outputFileHandler.write(docID + " " + numOfPostingsInStringFormat + "\n")
    currentLine = inputFileHandler.readline()
inputFileHandler.close()
outputFileHandler.close()
exit(1)

termDictFromFile1 = {}
inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/randomlySelectedWithUniversalAndLocalPostingIDs_20130924Night"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentTerm = lineElements[0]
    if currentTerm not in termDictFromFile1:
        termDictFromFile1[currentTerm] = 1
print "len(termDictFromFile1): ",len(termDictFromFile1)
inputFileHandler.close()

termDictFromFile2 = {}
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version1/termPieceInfoForRandomPostingTerms"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentTerm = lineElements[0]
    if currentTerm not in termDictFromFile2:
        termDictFromFile2[currentTerm] = 1
print "len(termDictFromFile2): ",len(termDictFromFile2)
inputFileHandler.close()

for currentTerm in termDictFromFile1:
    if currentTerm not in termDictFromFile2:
        print currentTerm,"NOT IN"
        exit(1)
print "Pass"
exit(1)

termDict = {}
inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/randomlySelectedWithUniversalAndLocalPostingIDs_20130924Night"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentTerm = line.strip().split(" ")[0]
    if currentTerm not in termDict:
        termDict[currentTerm] = 1
print "len(termDict): ",len(termDict)
inputFileHandler.close()
exit(1)

print "Program Begins..."
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version2_20140112Night/relRank_20140111Night_Probability.csv"
inputFileHandler = open(inputFileName,"r")
outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version2_20140112Night/relRank_20140111Night_Probability_toolkit_friendly"
outputFileHandler = open(outputFileName,"w")

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    classLabelPart1 = lineElements[0]
    for index,currentValue in enumerate(lineElements[4:]):
        outputLine = classLabelPart1 + "_" + str(index) + " " + currentValue + "\n"
        outputFileHandler.write(outputLine)

inputFileHandler.close()
outputFileHandler.close()
print "Overall:"
print "inputFileName: ",inputFileName
print "outputFileName: ",outputFileName
exit(1)

# outputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/queryTermPostingList_FromRandomlySelectedWithUniversalAndLocalPostingIDs_20130924Night"
# outputFileHandler = open(outputFileName,"w")

queryTermDict = {}
inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/queryTermsONLY_from_100K_queries_toolkit_compatible"
inputFileHandler = open(inputFileName1,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(":")
    currentTerm = lineElements[1]
    if currentTerm not in queryTermDict:
        queryTermDict[currentTerm] = 1
print "len(queryTermDict): ",len(queryTermDict)
inputFileHandler.close()

numOfPostingsCanBeComputed = 0
randomPostingTermDict = {}
inputFileName2 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/queryTermPostingList_FromRandomlySelectedWithUniversalAndLocalPostingIDs_20130924Night"
inputFileHandler = open(inputFileName2,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentTerm = lineElements[0]
    if currentTerm not in randomPostingTermDict:
        randomPostingTermDict[currentTerm] = 1
    if currentTerm in queryTermDict:
        numOfPostingsCanBeComputed += 1
        # outputFileHandler.write(line)
print "len(randomPostingTermDict): ",len(randomPostingTermDict)
print "numOfPostingsCanBeComputed: ",numOfPostingsCanBeComputed
inputFileHandler.close()
# outputFileHandler.close()

print "Overall Processing Statistics:"
print "inputFileName1: ",inputFileName1
print "inputFileName2: ",inputFileName2
# print "outputFileName: ",outputFileName
print "Program Ends."
exit(1)


randomPostingTermDict = {}
# option1:
# outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version1/termPieceInfoForQueryTerms"
# option2:
outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version1/termPieceInfoForRandomPostingTerms"
outputFileHandler = open(outputFileName,"w")

# option1:
# inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/queryTermsWithTheirLengthsOfInvertedList"
# option2:
inputFileName1 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/randomlySelectedWithUniversalAndLocalPostingIDs_20130924Night"
inputFileHandler = open(inputFileName1,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentTerm = lineElements[0]
    if currentTerm not in randomPostingTermDict:
        randomPostingTermDict[currentTerm] = 1
print "len(randomPostingTermDict):",len(randomPostingTermDict)
inputFileHandler.close()

inputFileName2 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithLengthOfListANDClassLabelANDNumOfPostingsInEachPieces20130819"
inputFileHandler = open(inputFileName2,"r")
lineCounter = 0

lineCounter += 1
currentLine = inputFileHandler.readline()
while currentLine:
    if lineCounter % 1000000 == 0:
        print lineCounter,"line processed."
    currentLineElements = currentLine.strip().split(" ")
    currentTerm = currentLineElements[0]
    if currentTerm in randomPostingTermDict:
        outputFileHandler.write(currentLine)
    lineCounter += 1
    currentLine = inputFileHandler.readline()
inputFileHandler.close()
outputFileHandler.close()

print "Overall Processing Statistics:"
print "inputFileName1:",inputFileName1
print "inputFileName2:",inputFileName2
print "outputFileName:",outputFileName
exit(1)


inputFileName = "/home/diaosi/Desktop/micro_cells_for_impact_score_ranges.csv"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/impactWithLengthAnalyze20131130Night/version3/predefinedClassLabelLowerBoundOfImpactScore20131221"
outputFileHandler = open(outputFileName,"w")

inputFileHandler.readline()
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    outputFileHandler.write(lineElements[0] + " " + lineElements[1] + "\n")
inputFileHandler.close()
outputFileHandler.close()
print "Overall Processing Statistics:"
print "inputFileName:",inputFileName
print "outputFileName:",outputFileName
exit(1)


inputFileName = "/home/diaosi/Desktop/micro_cells_for_list_length_ranges.csv"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/impactWithLengthAnalyze20131130Night/version3/predefinedClassLabelLowerBoundOfListLength20131221"
outputFileHandler = open(outputFileName,"w")

inputFileHandler.readline()
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    outputFileHandler.write(lineElements[0] + " " + lineElements[1] + "\n")
inputFileHandler.close()
outputFileHandler.close()
print "Overall Processing Statistics:"
print "inputFileName:",inputFileName
print "outputFileName:",outputFileName
exit(1)

'''
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/impactWithLengthAnalyze20131130Night/version3/quadTreeAreaIDWithProbabilities_combined_raw_20131220Night"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/impactWithLengthAnalyze20131130Night/version3/probabiliesBasedOnNumeratorANDDenominator_quadTreeProbabilities_20131221Night_toolkit_friendly"
outputFileHandler = open(outputFileName,"w")

twoDClassLabelWithProbabilityDict = {}

current2DclassLabel = "none_none"
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    leftBoundInIntFormat = int(lineElements[2])
    rightBoundInIntFormat = int(lineElements[3])
    upBoundInIntFormat = int(lineElements[4])
    downBoundInIntFormat = int(lineElements[5])
    probability = float(lineElements[-2])
    for i in range(leftBoundInIntFormat,rightBoundInIntFormat + 1 ):
        for j in range(upBoundInIntFormat,downBoundInIntFormat + 1):
            current2DclassLabel = str(i) + "_" + str(j)
            if current2DclassLabel not in twoDClassLabelWithProbabilityDict:
                twoDClassLabelWithProbabilityDict[current2DclassLabel] = probability
            else:
                print "critical error"
                exit(1)
print "len(twoDClassLabelWithProbabilityDict):",len(twoDClassLabelWithProbabilityDict)

for i in range(0,1306):
    for j in range(0,763):
        currentAccessKey = str(i) + "_" + str(j)
        if currentAccessKey not in twoDClassLabelWithProbabilityDict:
            print "critical error"
            print "currentAccessKey:",currentAccessKey
            exit(1)
        else:
            outputFileHandler.write(currentAccessKey + " " + str( twoDClassLabelWithProbabilityDict[currentAccessKey] ) + "\n")

outputFileHandler.close()
inputFileHandler.close()

print "Overall Processing Statistics:"
print "inputFileName:",inputFileName
print "outputFileName:",outputFileName
exit(1)
'''

'''
print "program begins..."
inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/randomlySampledPostingIDList20130828_with_set_of_probability_added_DEBUG"
inputFileHandler = open(inputFileName,"r")
minProbability = 10.0
maxProbability = 0.0
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    
    if maxProbability < float(lineElements[-1]):
         maxProbability = float(lineElements[-1])
    
    if minProbability > float(lineElements[-1]):
        minProbability = float(lineElements[-1])

print "maxProbability:",maxProbability
print "minProbability:",minProbability
inputFileHandler.close()
print "program ends."
exit(1)
'''

'''
print "program begins..."
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithLengthOfListANDClassLabelANDNumOfPostingsInEachPieces20130819"
inputFileHandler = open(inputFileName,"r")
outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithLengthOfList"
outputFileHandler = open(outputFileName,"w")

currentLine = inputFileHandler.readline()
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    outputFileHandler.write(currentLineElements[0] + " " + currentLineElements[1] + "\n")
    currentLine = inputFileHandler.readline()

outputFileHandler.close()
inputFileHandler.close()
print "Overall Processing Statistics"
print "inputFileName: ",inputFileName
print "outputFileName: ",outputFileName
print "program ends."
exit(1)
'''
